# -*- coding: utf-8 -*-
from scrapy.linkextractors import LinkExtractor
from girl.items.xp import *
import re
from datetime import *
from scrapy.http import Request as sreq
import platform
from scrapy_redis.spiders import RedisCrawlSpider
import redis

class xpSpider(RedisCrawlSpider):
    name = "xp"
    start_urls = [
        'http://1024.97cnlp.rocks/pw/',
    ]
    custom_settings = {
        "CONCURRENT_REQUESTS": 550,
        "HTTPCACHE_EXPIRATION_SECS": 60 * 60 * 24 * 365,
        "DEPTH_PRIORITY":1
    }
    pool = redis.ConnectionPool(host='localhost', port=6379)
    conn = redis.Redis(connection_pool=pool)
    conn.sadd(name+":start_urls", *start_urls)
    if platform.system() != 'Darwin':
        time = datetime.now().strftime('%Y-%m-%d-%H-%M')
        custom_settings["LOG_FILE"] = "/mnt/scrapy/crawler/%s-%s.log" % (name, time)

    def parse(self, response):
        url = response.css("#cate_1  span a::attr(href)").extract()
        print url
        def is_valid(x):
            if not re.search("ad/d{\d}",x):
                return True
            elif not re.search("thread.php?fid=8",x):
                return True
        for x in filter(is_valid, url):
            yield sreq(response.urljoin(x), callback=self.parse1,dont_filter=True)

    def parse1(self,response):
        i = 0
        for x in response.css('tr.tr3.t_one'):
            if i > 9:
                url = x.css("h3 a::attr(href)").extract_first()
                yield sreq(response.urljoin(url), callback=self.parse2)
            i += 1

    def parse2(self,response):
        item = xpItem()
        # import ipdb;ipdb.set_trace()
        item["title"] = response.css("#subject_tpc::text").extract_first()
        tmp = response.css(".tpc_content").extract_first()
        item["imgs"] = re.findall('img src="(.*?)"',tmp)
        item["seedurl"] = re.findall('href="(.*?)"',tmp)[-1]
        item["content"] = response.css(".tpc_content .f14").extract_first()
        item["url"] = response.url
        item["date"] = str(date.today())
        yield item



